#!/usr/bin/env python
#coding=utf-8
# author:wade
# contact: 317909531@qq.com
# datetime:2020/5/3 10:11

"""
文件说明：第一步，获取商品链接，存入数据库
"""
import os,django
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "shareAPI2020TB.settings")# project_name 项目名称
django.setup()

import logging
server_logger = logging.getLogger("server")

import re
import json

import requests
from lxml import etree
from tbitem import models


def getweb(configJson,num,fidNum):
    url = f'http://bbs.8080.net/forum-{fidNum}-{num}.html'
    # header 伪装浏览器设置
    print(url)
    headers = {
        'Cookie': configJson["cookie"]
    }
    session = requests.session()

    html = session.get(url,headers=headers)
    # print(html.text)
    return html

def readjson(jsonname):
    with open(jsonname, 'r',encoding='utf-8') as f:
        data = json.load(f)

    return data

def savejson(data,jsonname):
    # Writing JSON data
    with open(jsonname, 'w',encoding='utf-8') as f:
        json.dump(data, f,ensure_ascii=False,indent = 4)


def step1(fidNum):
    server_logger.error("step1-----Start")

    # 1.读取配置json，读取config.json
    configJson = {
  "填写说明": "----uid是bbs的作者id，cookie是bbs登录的cookie，这个程序运行一次，将会保存所有的发帖链接到作者json库",
  "uid": "334750",
  "cookie": "A2UO_2132_saltkey=ZXhKSThW; A2UO_2132_lastvisit=1588461375; A2UO_2132_sid=HLDQ2L; A2UO_2132_visitedfid=160; Hm_lvt_7c7fb80a5b246b349204dd1a7b719827=1588464991; A2UO_2132_forum_lastvisit=D_160_1588465379; A2UO_2132_viewid=tid_1117913; A2UO_2132_home_diymode=1; A2UO_2132_seccodeSAHLDQ2L0=5ef7i7CRqIyotseZ43M6cUFkcxIIHnMV%2F22lAY%2FxaxwPfSHFKf%2FaciNnKINdl8pNkL1oSnbPOe4jDgnmvHMC; A2UO_2132_ulastactivity=9c28%2BkdvQJb4OBRW2DLGifhJrVN6nLSlbM4FOCI4flEh3TXhigoQ; A2UO_2132_lastcheckfeed=354960%7C1588465641; A2UO_2132_auth=21c5yqHvDLI%2BX7F0tYfvK294yMtz1YfxF3DCzR%2Fk1r4blaXmrE4q9VMYrcIfnwPO5mJVwA1JSjrZG1x1BSONP%2FGF%2F0c; A2UO_2132_noticeTitle=1; A2UO_2132_home_readfeed=1588465653; A2UO_2132_nofavfid=1; A2UO_2132_sendmail=1; A2UO_2132_lastact=1588465782%09home.php%09spacecp; A2UO_2132_checkpm=1; Hm_lpvt_7c7fb80a5b246b349204dd1a7b719827=1588465783"}
    print(configJson)
    for page in range(1,200):
        html = getweb(configJson, page,fidNum=fidNum)
        # 3.etree获取内容
        '//*[@id="uhd"]/div[2]/h2'
        etreeHtml = etree.HTML(html.text)

        # etree抓作每一行的标题内容
        zuozhe = etreeHtml.xpath('//tbody[contains(@id,"normalthread_")]/tr/th/a')
        for i in zuozhe:
            # print(i.tag)  #标签类型
            print(i.attrib['href'])  # 链接 link
            print(i.text)  # 标题
            # 把内容存进数据库BBsLinks  （link  title status--1）
            match = re.compile(rf'thread-(\d+)-1-{page}.html')
            linkNum = match.findall(i.attrib['href'])
            print(linkNum)

            queryNum = models.BBsLinks.objects.filter(linkNum = linkNum[0]).first()
            if queryNum:  #存在就print
                server_logger.error(f"查询到linkNum为{linkNum}---已存在，step1结束")
                print('已经存在')
                return
            else:
                link = models.BBsLinks.objects.create(link=i.attrib['href'], title=i.text, linkNum=linkNum[0])
        server_logger.error(f"step1-----第{page}页运行成功{len(zuozhe)}条数据")

if __name__ == '__main__':
    step1(160)  #负责爬取bbs的标题内容，抓入库BBsLinks